
import os
NOVA_HOME = "/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA"
NOVA_DATA_HOME = '/home/labs/hornsteinlab/Collaboration/FUNOVA'

LOGS_PATH = os.path.join(NOVA_DATA_HOME, "outputs/preprocessing/logs/")
PLOT_PATH = os.path.join(NOVA_DATA_HOME, "outputs/logs/")

os.chdir(NOVA_HOME)

import pandas as pd
import contextlib
import io
from IPython.display import display, Javascript
import seaborn as sns
from tools.preprocessing_tools.qc_reports.qc_utils import log_files_qc, run_validate_folder_structure, display_diff, sample_and_calc_variance, \
                                                show_site_survival_dapi_brenner, show_site_survival_dapi_cellpose, \
                                                show_site_survival_dapi_tiling, show_site_survival_target_brenner, \
                                                calc_total_sums, plot_filtering_heatmap, show_total_sum_tables, \
                                                plot_cell_count, plot_catplot, plot_hm_combine_batches, plot_hm, \
                                                run_calc_hist_new, plot_marker_data, find_bad_wells, create_marker_info_df
                                                
from tools.preprocessing_tools.qc_reports.qc_config import (
    funova_markers as markers,
    funova_cell_lines as cell_lines,
    funova_cell_lines_to_cond as cell_lines_to_cond,
    funova_cell_lines_for_disp as cell_lines_for_disp,
    funova_reps as reps,
    funova_line_colors as line_colors,
    funova_lines_order as lines_order,
    funova_custom_palette as custom_palette,
    funova_expected_dapi_raw as expected_dapi_raw,
    funova_panels as panels,
    funova_marker_info as marker_info
)

from tools.show_images_utils import *
%load_ext autoreload
%autoreload 2


# choose batches
batches = ['Batch1', 'Batch2']#, 'batch2', 'batch3']
batches

['Batch1', 'Batch2']


validate_antibody = False


df = log_files_qc(LOGS_PATH, batches, only_wt_cond = False)

reading logs of Batch1

Total of 1 files were read.
Before dup handeling  (180333, 21)
After duplication removal #1: (180333, 22)
After duplication removal #2: (180333, 22)


df[['cell_line', 'marker', 'cell_line_cond']] = df[['cell_line', 'marker', 'cell_line_cond']].apply(lambda x: x.str.replace('_', '-', regex=False))


df['filename'] = df['filename'].str.split('-').str[0]
df['site_num'] = df['site_num'].str.split('-').str[0]


df_dapi = df[df.marker=='DAPI']
df_target = df[df.marker!='DAPI']


root_directory_raw = os.path.join(NOVA_DATA_HOME, 'input', 'images', 'raw')


df_metadata = extract_image_metadata(root_directory_raw, FILE_EXTENSION = '.tiff', KEY_BATCH = 'Batch')


# root_path = "/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/"
# marker_info_df = create_marker_info_df(root_path)


batches_raw = [batch.replace("_16bit_no_downsample","") for batch in batches]
raws = run_validate_folder_structure(root_directory_raw, False, panels, markers.copy(), PLOT_PATH, marker_info,
                                    cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
                                     batches=batches_raw, fig_width=8,fig_height = 40,
                                    expected_count=100, validate_antibody = validate_antibody)

Batch1
Folder structure is valid.
No bad files are found.
Total Sites:  118400

========
Batch2
Folder structure is valid.
No bad files are found.
Total Sites:  118400

========
====================


root_directory_proc = os.path.join(NOVA_DATA_HOME, 'input', 'images', 'processed')
procs = run_validate_folder_structure(root_directory_proc, True, panels, markers, PLOT_PATH, marker_info,
                                    cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
                                     batches=batches, fig_width=8,fig_height = 40,
                                    expected_count=100, validate_antibody = validate_antibody)

Batch1
Folder structure is valid.
No bad files are found.
Total Sites:  89058

========
Batch2
Folder structure is valid.
No bad files are found.
Total Sites:  72173

========
====================


display_diff(batches, raws, procs, PLOT_PATH, fig_width=8, fig_height = 40)

Batch1

========
Batch2

========


for batch in batches:
    with contextlib.redirect_stdout(io.StringIO()):
        var = sample_and_calc_variance(root_directory_proc, batch, 
                                       sample_size_per_markers=200, cond_count=1, rep_count=len(reps), 
                                       num_markers=len(markers))
    print(f'{batch} var: ',var)

Batch1 var:  0.02010689204799957
Batch2 var:  0.02211178779242227


dapi_filter_by_brenner = show_site_survival_dapi_brenner(df_dapi,batches, line_colors, panels,
                                                        figsize=(6,18), reps=reps, vmax=100)


show_images(df_metadata, marker='DAPI', show_DAPI=False, batch=2, panel='L', cell_line='C9orf72-HRE-1008566', rep=1,
            condition='stress', funova=True, samples=5)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

1
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f61-ch1t1.tiff
2
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f07-ch1t1.tiff
3
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f81-ch1t1.tiff
4
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f01-ch1t1.tiff
5
Img shape: (1024, 1024)


show_images(df_metadata, marker='DAPI', show_DAPI=False, batch=2, panel='D', cell_line='Control-1017118', rep=1,
            condition='stress', funova=True, samples=5)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

1
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f66-ch1t1.tiff
2
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f59-ch1t1.tiff
3
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f26-ch1t1.tiff
4
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f67-ch1t1.tiff
5
Img shape: (1024, 1024)


df_metadata

dfb


temp1 = get_specific_imgs(dfb, marker='DAPI', batch=2,cell_line='Control_1017118', rep=1, condition='stress')
temp = temp1.loc[temp1.Panel =='D'].Target_Sharpness_Brenner
temp1 = temp1.Target_Sharpness_Brenner


plt.hist(temp1)
plt.hist(temp)

(array([ 7.,  0.,  1.,  0.,  0.,  0.,  3.,  3.,  5., 81.]),
 array([  185.60997523,  9922.55692795, 19659.50388067, 29396.45083339,
        39133.39778611, 48870.34473883, 58607.29169155, 68344.23864427,
        78081.18559699, 87818.13254971, 97555.07950243]),
 <BarContainer object of 10 artists>)


dapi_filter_by_cellpose = show_site_survival_dapi_cellpose(df_dapi, batches, dapi_filter_by_brenner, 
                                                           line_colors, panels, figsize=(6,18), reps=reps)


dapi_filter_by_tiling=show_site_survival_dapi_tiling(df_dapi, batches, dapi_filter_by_cellpose, 
                                                     line_colors, panels, figsize=(6,18), reps=reps)


show_site_survival_target_brenner(df_dapi, df_target, dapi_filter_by_tiling,
                                 figsize=(6,24), markers=markers)


## Images that were completely removed and their corresponding DAPI 
show_images(df_metadata, marker = 'Senescence-signaling', show_DAPI = True, 
            batch = 1, cell_line ='Control-1048087', 
            condition = 'stress', rep=2, funova = True, samples = 5)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

1
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f54-ch3t1.tiff
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f54-ch1t1.tiff
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

--------------------------------
2
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f03-ch3t1.tiff
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f03-ch1t1.tiff
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

--------------------------------
3
Img shape: (1024, 1024)


show_images(df_metadata, marker='Parthanatos-early', show_DAPI=False, batch=2, panel='H', cell_line='TDP--43-G348V-1057052', 
            rep=2, condition='Untreated', funova=True, samples=5)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

1
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f12-ch2t1.tiff
2
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f18-ch2t1.tiff
3
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f37-ch2t1.tiff
4
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f05-ch2t1.tiff
5
Img shape: (1024, 1024)


show_images(df_metadata, marker='Protein-degradation', show_DAPI=False, batch=2, panel='L', cell_line='C9orf72-HRE-981344', rep=1,
            condition='stress', funova=True, samples=5)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

1
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f53-ch2t1.tiff
2
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f78-ch2t1.tiff
3
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f14-ch2t1.tiff
4
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f77-ch2t1.tiff
5
Img shape: (1024, 1024)


dfb = pd.read_csv("/home/labs/hornsteinlab/Collaboration/FUNOVA/outputs/preprocessing/brenner/raw_metrics280125_exp3.csv")
# Extract the panel (e.g., "panelD") from the Path column
dfb['Panel'] = dfb['Path'].str.extract(r'/panel([A-Z])/', expand=True)
dfb['image_id'] = dfb['Path'].apply(lambda path: path.split('/')[-1].split('.')[0])
find_bad_wells(dfb, threshold=50000, percentage_filter=70)


thresholds = {
    "Stress_initiation": (0, 99),
    "Aberrant_splicing": (1, 97),
    "Autophagy": (5, 97),
    "DAPI": (0.2, 95),
    "Apoptosis": (0.27, 95),
    "impaired_Autophagosome": (5, 99),
    "Cytoskeleton": (10, 99.8),
    "DNA_damage_P53BP1": (0.3, 90),
    "DNA_damage_pH2Ax": (7, 85),
    "mature_Autophagosome": (2, 98),
    "Necrosis": (4, 95),
    "Neuronal_activity": (5, 90),
    "Nuclear_speckles_SC35": (0.2, 85),
    "Nuclear_speckles_SON": (0.2, 85),
    "Parthanatos_early": (0, 85),
    "Parthanatos_late": (5, 95),
    "Protein_degradation": (5, 92),
    "Senescence_signaling": (10, 90),
    "Splicing_factories": (0, 95),
    "TDP-43": (5, 99.2),
    "Ubiquitin_levels": (2, 90),
    "UPR_ATF4": (2, 99.7), 
    "UPR_ATF6": (3, 100),
    "UPR_IRE1a": (2, 98),
    "Necroptosis_pMLKL": (5, 90),
    "Necroptosis_HMGB1": (0.01, 90),
}
# Extended batch data with new entries
data = [
    (2, 'DNA_damage_pH2Ax', 1, 1017118, 'Untreated'),
    (2, 'Parthanatos_early', 1, 1048087, 'stress'),
    (2, 'DNA_damage_pH2Ax', 2, 1008566, 'stress'),
    (2, 'Parthanatos_early', 1, 981344, 'stress'),
    (2, 'Nuclear_speckles_SON', 1, 981344, 'stress'),
    (1, 'Autophagy', 1, 1057052, 'stress'),
    (1, 'impaired_Autophagosome', 1, 1057052, 'stress'),
    (1, 'UPR_IRE1a', 1, 1057052, 'stress'),
    (1, 'Ubiquitin_levels', 1, 1057052, 'stress'),
    (1, 'UPR_IRE1a', 1, 981344, 'stress'),
    (1, 'Ubiquitin_levels', 1, 981344, 'stress'),
    (1, 'DNA_damage_P53BP1', 1, 981344, 'stress'),
    (1, 'Neuronal_activity', 1, 981344, 'stress'),
    (1, 'Necroptosis_HMGB1', 1, 981344, 'stress'),
    (1, 'Necrosis', 1, 981344, 'stress'),
    (1, 'Nuclear_speckles_SC35', 2, 981344, 'stress'),
    (1, 'Splicing_factories', 2, 981344, 'stress'),
    (1, 'Aberrant_splicing', 2, 981344, 'stress'),
    (1, 'Parthanatos_late', 2, 981344, 'stress'),
    (1, 'Senescence_signaling', 2, 1048087, 'stress'),
    (1, 'Apoptosis', 2, 981344, 'stress'),
    (1, 'Necroptosis_pMLKL', 2, 981344, 'stress'),
    (2, 'Autophagy', 1, 1017118, 'stress'),
    (2, 'impaired_Autophagosome', 1, 1017118, 'stress'),
    (2, 'UPR_IRE1a', 1, 1048087, 'Untreated'),
    (2, 'UPR_IRE1a', 1, 1008566, 'stress'),
    (2, 'Ubiquitin_levels', 1, 1048087, 'Untreated'),
    (2, 'Ubiquitin_levels', 1, 1008566, 'stress'),
    (2, 'DNA_damage_P53BP1', 1, 1017118, 'stress'),
    (2, 'Neuronal_activity', 1, 1017118, 'stress'),
    (2, 'DNA_damage_P53BP1', 2, 981344, 'stress'),
    (2, 'Neuronal_activity', 2, 981344, 'stress'),
    (2, 'DNA_damage_P53BP1', 1, 981344, 'stress'),
    (2, 'Neuronal_activity', 1, 981344, 'stress'),
    (2, 'DNA_damage_P53BP1', 1, 1057052, 'stress'),
    (2, 'Neuronal_activity', 1, 1057052, 'stress'),
    (2, 'DNA_damage_P53BP1', 1, 1005373, 'stress'),
    (2, 'Neuronal_activity', 1, 1005373, 'stress'),
    (2, 'DNA_damage_P53BP1', 2, 1057052, 'stress'),
    (2, 'Neuronal_activity', 2, 1057052, 'stress'),
    (2, 'Necroptosis_HMGB1', 1, 981344, 'Untreated'),
    (2, 'Necrosis', 1, 981344, 'Untreated'),
    (2, 'Necroptosis_HMGB1', 2, 981344, 'stress'),
    (2, 'Necrosis', 2, 981344, 'stress'),
    (2, 'Necroptosis_HMGB1', 2, 1001733, 'stress'),
    (2, 'Necrosis', 2, 1001733, 'stress'),
    (2, 'Necroptosis_HMGB1', 2, 1005373, 'stress'),
    (2, 'Necrosis', 2, 1005373, 'stress')
]

# Cell line ID mapping
cell_line_mapping = {
    1008566: 'C9orf72-HRE_1008566',
    981344: 'C9orf72-HRE_981344',
    1001733: 'Control_1001733',
    1017118: 'Control_1017118',
    1025045: 'Control_1025045',
    1048087: 'Control_1048087',
    1057052: 'TDP--43-G348V_1057052',
    1005373: 'TDP--43-N390D_1005373'
}

# Convert to DataFrame (renamed to `marker_df`)
marker_df = pd.DataFrame(data, columns=['Batch', 'Name', 'Rep', 'CellLineID', 'Condition'])

lower_thresh_DAPI, upper_thresh_DAPI = thresholds['DAPI']
cb = get_specific_imgs(dfb, marker='DAPI')
cb = cb.loc[(cb.Batch == 'Batch1') | (cb.Batch == 'Batch2')].Target_Sharpness_Brenner.values
lower_p_DAPI = np.percentile(cb, lower_thresh_DAPI)
upper_p_DAPI = np.percentile(cb, upper_thresh_DAPI)

# Iterate over each row
for _, row in marker_df.iterrows():
    batch, marker, rep, cell_line_id, condition = row

    # Match Cell Line Name
    cell_line = cell_line_mapping.get(cell_line_id, f'Unknown_{cell_line_id}')

    # Fetch images based on specific conditions
    cb1 = get_specific_imgs(dfb, marker=marker, batch=batch, rep=rep, 
                           cell_line=cell_line, condition=condition)
    
    dapi_paths = cb1.Path.apply(lambda p: get_dapi_path(p, marker1=marker, marker2='DAPI', funova=True))
    dfDAPI = dfb.loc[dfb['Path'].isin(dapi_paths)].Target_Sharpness_Brenner.values
    cb1 = cb1.Target_Sharpness_Brenner.values

    # Print statistics dynamically for the specific selection
    print(f"Stats for marker: {marker}, batch: {batch}, rep: {rep}, cell_line: {cell_line}, condition: {condition}")
    print(f"Mean: {np.mean(cb1):.2f}, Median: {np.median(cb1):.2f}, "
          f"10th Percentile: {np.percentile(cb1, 10):.2f}, "
          f"50th Percentile (Median): {np.percentile(cb1, 50):.2f}, "
          f"90th Percentile: {np.percentile(cb1, 90):.2f}")

    # Fetch images for the entire dataset with the given marker
    cb = get_specific_imgs(dfb, marker=marker)
    cb = cb.loc[(cb.Batch == 'Batch1') | (cb.Batch == 'Batch2')].Target_Sharpness_Brenner.values
    
    lower_thresh, upper_thresh = thresholds[marker]
    lower_p = np.percentile(cb, lower_thresh)
    upper_p = np.percentile(cb, upper_thresh)
    
    filtered_out = np.sum((cb1 < lower_p) | (cb1 > upper_p))
    total_count = len(cb1)
    filtered_percentage = (filtered_out / total_count) * 100 if total_count > 0 else 0
    
    filtered_out_DAPI = np.sum((dfDAPI < lower_p_DAPI) | (dfDAPI > upper_p_DAPI))
    total_count_DAPI = len(dfDAPI)
    filtered_percentage_DAPI = (filtered_out_DAPI / total_count_DAPI) * 100 if total_count_DAPI > 0 else 0

    # Print statistics dynamically for the full dataset
    print(f"Stats for marker: {marker} (All batches, reps, and cell lines)")
    print(f"Mean: {np.mean(cb):.2f}, Median: {np.median(cb):.2f}, "
          f"10th Percentile: {np.percentile(cb, 10):.2f}, "
          f"50th Percentile (Median): {np.percentile(cb, 50):.2f}, "
          f"90th Percentile: {np.percentile(cb, 90):.2f}")
    
    print(f"Threshold Percentiles ({lower_thresh}% - {upper_thresh}%): {lower_p:.2f} - {upper_p:.2f}")
    print(f"Filtered Out: {filtered_out}/{total_count} ({filtered_percentage:.2f}%)")
    print(f"Filtered Out_DAPI: {filtered_out_DAPI}/{total_count_DAPI} ({filtered_percentage_DAPI:.2f}%)")

    print("-" * 80)  # Separator for readability

Stats for marker: DNA_damage_pH2Ax, batch: 2, rep: 1, cell_line: Control_1017118, condition: Untreated
Mean: 18838.00, Median: 20707.26, 10th Percentile: 10668.05, 50th Percentile (Median): 20707.26, 90th Percentile: 25411.50
Stats for marker: DNA_damage_pH2Ax (All batches, reps, and cell lines)
Mean: 8754.40, Median: 5613.45, 10th Percentile: 1402.97, 50th Percentile (Median): 5613.45, 90th Percentile: 14670.41
Threshold Percentiles (7% - 85%): 880.46 - 11278.07
Filtered Out: 87/100 (87.00%)
Filtered Out_DAPI: 1/100 (1.00%)
--------------------------------------------------------------------------------
Stats for marker: Parthanatos_early, batch: 2, rep: 1, cell_line: Control_1048087, condition: stress
Mean: 4417.15, Median: 3275.69, 10th Percentile: 1385.82, 50th Percentile (Median): 3275.69, 90th Percentile: 10353.77
Stats for marker: Parthanatos_early (All batches, reps, and cell lines)
Mean: 5137.73, Median: 1356.69, 10th Percentile: 435.16, 50th Percentile (Median): 1356.69, 90th Percentile: 7115.26
Threshold Percentiles (0% - 85%): 76.89 - 3000.88
Filtered Out: 58/100 (58.00%)
Filtered Out_DAPI: 11/100 (11.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_pH2Ax, batch: 2, rep: 2, cell_line: C9orf72-HRE_1008566, condition: stress
Mean: 2744.98, Median: 306.88, 10th Percentile: 138.35, 50th Percentile (Median): 306.88, 90th Percentile: 5602.55
Stats for marker: DNA_damage_pH2Ax (All batches, reps, and cell lines)
Mean: 8754.40, Median: 5613.45, 10th Percentile: 1402.97, 50th Percentile (Median): 5613.45, 90th Percentile: 14670.41
Threshold Percentiles (7% - 85%): 880.46 - 11278.07
Filtered Out: 66/100 (66.00%)
Filtered Out_DAPI: 17/100 (17.00%)
--------------------------------------------------------------------------------
Stats for marker: Parthanatos_early, batch: 2, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 25628.22, Median: 17188.75, 10th Percentile: 1701.12, 50th Percentile (Median): 17188.75, 90th Percentile: 77173.85
Stats for marker: Parthanatos_early (All batches, reps, and cell lines)
Mean: 5137.73, Median: 1356.69, 10th Percentile: 435.16, 50th Percentile (Median): 1356.69, 90th Percentile: 7115.26
Threshold Percentiles (0% - 85%): 76.89 - 3000.88
Filtered Out: 79/100 (79.00%)
Filtered Out_DAPI: 5/100 (5.00%)
--------------------------------------------------------------------------------
Stats for marker: Nuclear_speckles_SON, batch: 2, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 35049.12, Median: 22153.54, 10th Percentile: 1366.38, 50th Percentile (Median): 22153.54, 90th Percentile: 89777.08
Stats for marker: Nuclear_speckles_SON (All batches, reps, and cell lines)
Mean: 8487.77, Median: 1007.12, 10th Percentile: 542.70, 50th Percentile (Median): 1007.12, 90th Percentile: 25995.89
Threshold Percentiles (0.2% - 85%): 281.61 - 4041.04
Filtered Out: 76/100 (76.00%)
Filtered Out_DAPI: 2/100 (2.00%)
--------------------------------------------------------------------------------
Stats for marker: Autophagy, batch: 1, rep: 1, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 10093.72, Median: 531.17, 10th Percentile: 157.05, 50th Percentile (Median): 531.17, 90th Percentile: 15555.22
Stats for marker: Autophagy (All batches, reps, and cell lines)
Mean: 2324.85, Median: 1347.71, 10th Percentile: 308.42, 50th Percentile (Median): 1347.71, 90th Percentile: 3511.05
Threshold Percentiles (5% - 97%): 222.36 - 5255.08
Filtered Out: 57/100 (57.00%)
Filtered Out_DAPI: 20/100 (20.00%)
--------------------------------------------------------------------------------
Stats for marker: impaired_Autophagosome, batch: 1, rep: 1, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 7503.05, Median: 1186.66, 10th Percentile: 633.22, 50th Percentile (Median): 1186.66, 90th Percentile: 13287.96
Stats for marker: impaired_Autophagosome (All batches, reps, and cell lines)
Mean: 2689.62, Median: 1989.26, 10th Percentile: 852.36, 50th Percentile (Median): 1989.26, 90th Percentile: 3844.37
Threshold Percentiles (5% - 99%): 506.44 - 18506.37
Filtered Out: 17/100 (17.00%)
Filtered Out_DAPI: 20/100 (20.00%)
--------------------------------------------------------------------------------
Stats for marker: UPR_IRE1a, batch: 1, rep: 1, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 12446.03, Median: 1629.60, 10th Percentile: 691.88, 50th Percentile (Median): 1629.60, 90th Percentile: 59199.03
Stats for marker: UPR_IRE1a (All batches, reps, and cell lines)
Mean: 3151.35, Median: 2478.02, 10th Percentile: 1032.42, 50th Percentile (Median): 2478.02, 90th Percentile: 4380.11
Threshold Percentiles (2% - 98%): 500.31 - 7289.88
Filtered Out: 26/100 (26.00%)
Filtered Out_DAPI: 26/100 (26.00%)
--------------------------------------------------------------------------------
Stats for marker: Ubiquitin_levels, batch: 1, rep: 1, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 31823.53, Median: 12149.22, 10th Percentile: 5346.91, 50th Percentile (Median): 12149.22, 90th Percentile: 97827.21
Stats for marker: Ubiquitin_levels (All batches, reps, and cell lines)
Mean: 5765.68, Median: 4180.66, 10th Percentile: 1406.83, 50th Percentile (Median): 4180.66, 90th Percentile: 8203.24
Threshold Percentiles (2% - 90%): 661.91 - 8203.24
Filtered Out: 66/100 (66.00%)
Filtered Out_DAPI: 26/100 (26.00%)
--------------------------------------------------------------------------------
Stats for marker: UPR_IRE1a, batch: 1, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 6038.85, Median: 2291.70, 10th Percentile: 1186.16, 50th Percentile (Median): 2291.70, 90th Percentile: 15993.96
Stats for marker: UPR_IRE1a (All batches, reps, and cell lines)
Mean: 3151.35, Median: 2478.02, 10th Percentile: 1032.42, 50th Percentile (Median): 2478.02, 90th Percentile: 4380.11
Threshold Percentiles (2% - 98%): 500.31 - 7289.88
Filtered Out: 25/100 (25.00%)
Filtered Out_DAPI: 21/100 (21.00%)
--------------------------------------------------------------------------------
Stats for marker: Ubiquitin_levels, batch: 1, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 32535.45, Median: 13961.59, 10th Percentile: 2585.12, 50th Percentile (Median): 13961.59, 90th Percentile: 92073.41
Stats for marker: Ubiquitin_levels (All batches, reps, and cell lines)
Mean: 5765.68, Median: 4180.66, 10th Percentile: 1406.83, 50th Percentile (Median): 4180.66, 90th Percentile: 8203.24
Threshold Percentiles (2% - 90%): 661.91 - 8203.24
Filtered Out: 68/100 (68.00%)
Filtered Out_DAPI: 21/100 (21.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 1, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 83518.02, Median: 94207.86, 10th Percentile: 53655.27, 50th Percentile (Median): 94207.86, 90th Percentile: 98298.86
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 97/100 (97.00%)
Filtered Out_DAPI: 65/100 (65.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 1, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 72466.41, Median: 86382.04, 10th Percentile: 25235.22, 50th Percentile (Median): 86382.04, 90th Percentile: 94564.54
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 99/100 (99.00%)
Filtered Out_DAPI: 65/100 (65.00%)
--------------------------------------------------------------------------------
Stats for marker: Necroptosis_HMGB1, batch: 1, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 56228.74, Median: 58945.84, 10th Percentile: 12549.80, 50th Percentile (Median): 58945.84, 90th Percentile: 98325.30
Stats for marker: Necroptosis_HMGB1 (All batches, reps, and cell lines)
Mean: 6658.95, Median: 824.32, 10th Percentile: 326.90, 50th Percentile (Median): 824.32, 90th Percentile: 14553.35
Threshold Percentiles (0.01% - 90%): 116.38 - 14553.35
Filtered Out: 86/100 (86.00%)
Filtered Out_DAPI: 47/100 (47.00%)
--------------------------------------------------------------------------------
Stats for marker: Necrosis, batch: 1, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 31248.68, Median: 2777.40, 10th Percentile: 1073.19, 50th Percentile (Median): 2777.40, 90th Percentile: 93448.21
Stats for marker: Necrosis (All batches, reps, and cell lines)
Mean: 4499.97, Median: 1645.52, 10th Percentile: 896.73, 50th Percentile (Median): 1645.52, 90th Percentile: 3915.14
Threshold Percentiles (4% - 95%): 636.23 - 7107.17
Filtered Out: 42/100 (42.00%)
Filtered Out_DAPI: 47/100 (47.00%)
--------------------------------------------------------------------------------
Stats for marker: Nuclear_speckles_SC35, batch: 1, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 72938.76, Median: 87505.81, 10th Percentile: 23095.22, 50th Percentile (Median): 87505.81, 90th Percentile: 95186.46
Stats for marker: Nuclear_speckles_SC35 (All batches, reps, and cell lines)
Mean: 6544.52, Median: 957.65, 10th Percentile: 534.14, 50th Percentile (Median): 957.65, 90th Percentile: 12064.62
Threshold Percentiles (0.2% - 85%): 258.82 - 2543.98
Filtered Out: 99/100 (99.00%)
Filtered Out_DAPI: 49/100 (49.00%)
--------------------------------------------------------------------------------
Stats for marker: Splicing_factories, batch: 1, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 54112.27, Median: 49446.25, 10th Percentile: 8593.07, 50th Percentile (Median): 49446.25, 90th Percentile: 100212.20
Stats for marker: Splicing_factories (All batches, reps, and cell lines)
Mean: 2978.87, Median: 626.68, 10th Percentile: 276.03, 50th Percentile (Median): 626.68, 90th Percentile: 1899.90
Threshold Percentiles (0% - 95%): 111.23 - 5836.77
Filtered Out: 91/100 (91.00%)
Filtered Out_DAPI: 49/100 (49.00%)
--------------------------------------------------------------------------------
Stats for marker: Aberrant_splicing, batch: 1, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 27932.63, Median: 13144.94, 10th Percentile: 7882.15, 50th Percentile (Median): 13144.94, 90th Percentile: 87572.43
Stats for marker: Aberrant_splicing (All batches, reps, and cell lines)
Mean: 8166.03, Median: 6349.42, 10th Percentile: 2695.15, 50th Percentile (Median): 6349.42, 90th Percentile: 14128.50
Threshold Percentiles (1% - 97%): 242.60 - 19755.72
Filtered Out: 32/100 (32.00%)
Filtered Out_DAPI: 34/100 (34.00%)
--------------------------------------------------------------------------------
Stats for marker: Parthanatos_late, batch: 1, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 19081.00, Median: 3941.15, 10th Percentile: 1283.63, 50th Percentile (Median): 3941.15, 90th Percentile: 77362.24
Stats for marker: Parthanatos_late (All batches, reps, and cell lines)
Mean: 3836.63, Median: 2591.29, 10th Percentile: 1184.37, 50th Percentile (Median): 2591.29, 90th Percentile: 6267.01
Threshold Percentiles (5% - 95%): 914.30 - 7237.04
Filtered Out: 45/100 (45.00%)
Filtered Out_DAPI: 34/100 (34.00%)
--------------------------------------------------------------------------------
Stats for marker: Senescence_signaling, batch: 1, rep: 2, cell_line: Control_1048087, condition: stress
Mean: 23694.13, Median: 23690.13, 10th Percentile: 20558.55, 50th Percentile (Median): 23690.13, 90th Percentile: 27428.32
Stats for marker: Senescence_signaling (All batches, reps, and cell lines)
Mean: 7948.93, Median: 2798.33, 10th Percentile: 1349.50, 50th Percentile (Median): 2798.33, 90th Percentile: 8807.32
Threshold Percentiles (10% - 90%): 1349.50 - 8807.32
Filtered Out: 99/100 (99.00%)
Filtered Out_DAPI: 12/100 (12.00%)
--------------------------------------------------------------------------------
Stats for marker: Apoptosis, batch: 1, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 35619.88, Median: 2481.69, 10th Percentile: 393.72, 50th Percentile (Median): 2481.69, 90th Percentile: 102597.95
Stats for marker: Apoptosis (All batches, reps, and cell lines)
Mean: 4833.76, Median: 932.10, 10th Percentile: 546.90, 50th Percentile (Median): 932.10, 90th Percentile: 2526.15
Threshold Percentiles (0.27% - 95%): 252.85 - 13142.16
Filtered Out: 40/100 (40.00%)
Filtered Out_DAPI: 41/100 (41.00%)
--------------------------------------------------------------------------------
Stats for marker: Necroptosis_pMLKL, batch: 1, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 37191.21, Median: 5457.77, 10th Percentile: 2601.01, 50th Percentile (Median): 5457.77, 90th Percentile: 104876.88
Stats for marker: Necroptosis_pMLKL (All batches, reps, and cell lines)
Mean: 7252.56, Median: 4269.27, 10th Percentile: 2534.47, 50th Percentile (Median): 4269.27, 90th Percentile: 7690.68
Threshold Percentiles (5% - 90%): 1408.56 - 7690.68
Filtered Out: 41/100 (41.00%)
Filtered Out_DAPI: 41/100 (41.00%)
--------------------------------------------------------------------------------
Stats for marker: Autophagy, batch: 2, rep: 1, cell_line: Control_1017118, condition: stress
Mean: 28492.27, Median: 23594.09, 10th Percentile: 2303.97, 50th Percentile (Median): 23594.09, 90th Percentile: 69949.43
Stats for marker: Autophagy (All batches, reps, and cell lines)
Mean: 2324.85, Median: 1347.71, 10th Percentile: 308.42, 50th Percentile (Median): 1347.71, 90th Percentile: 3511.05
Threshold Percentiles (5% - 97%): 222.36 - 5255.08
Filtered Out: 82/100 (82.00%)
Filtered Out_DAPI: 10/100 (10.00%)
--------------------------------------------------------------------------------
Stats for marker: impaired_Autophagosome, batch: 2, rep: 1, cell_line: Control_1017118, condition: stress
Mean: 22885.30, Median: 15687.97, 10th Percentile: 887.39, 50th Percentile (Median): 15687.97, 90th Percentile: 57051.98
Stats for marker: impaired_Autophagosome (All batches, reps, and cell lines)
Mean: 2689.62, Median: 1989.26, 10th Percentile: 852.36, 50th Percentile (Median): 1989.26, 90th Percentile: 3844.37
Threshold Percentiles (5% - 99%): 506.44 - 18506.37
Filtered Out: 54/100 (54.00%)
Filtered Out_DAPI: 10/100 (10.00%)
--------------------------------------------------------------------------------
Stats for marker: UPR_IRE1a, batch: 2, rep: 1, cell_line: Control_1048087, condition: Untreated
Mean: 14329.68, Median: 1444.15, 10th Percentile: 933.28, 50th Percentile (Median): 1444.15, 90th Percentile: 63322.41
Stats for marker: UPR_IRE1a (All batches, reps, and cell lines)
Mean: 3151.35, Median: 2478.02, 10th Percentile: 1032.42, 50th Percentile (Median): 2478.02, 90th Percentile: 4380.11
Threshold Percentiles (2% - 98%): 500.31 - 7289.88
Filtered Out: 27/100 (27.00%)
Filtered Out_DAPI: 3/100 (3.00%)
--------------------------------------------------------------------------------
Stats for marker: UPR_IRE1a, batch: 2, rep: 1, cell_line: C9orf72-HRE_1008566, condition: stress
Mean: 3678.84, Median: 1119.25, 10th Percentile: 660.99, 50th Percentile (Median): 1119.25, 90th Percentile: 6801.02
Stats for marker: UPR_IRE1a (All batches, reps, and cell lines)
Mean: 3151.35, Median: 2478.02, 10th Percentile: 1032.42, 50th Percentile (Median): 2478.02, 90th Percentile: 4380.11
Threshold Percentiles (2% - 98%): 500.31 - 7289.88
Filtered Out: 16/100 (16.00%)
Filtered Out_DAPI: 16/100 (16.00%)
--------------------------------------------------------------------------------
Stats for marker: Ubiquitin_levels, batch: 2, rep: 1, cell_line: Control_1048087, condition: Untreated
Mean: 9994.41, Median: 1666.91, 10th Percentile: 770.08, 50th Percentile (Median): 1666.91, 90th Percentile: 35706.55
Stats for marker: Ubiquitin_levels (All batches, reps, and cell lines)
Mean: 5765.68, Median: 4180.66, 10th Percentile: 1406.83, 50th Percentile (Median): 4180.66, 90th Percentile: 8203.24
Threshold Percentiles (2% - 90%): 661.91 - 8203.24
Filtered Out: 23/100 (23.00%)
Filtered Out_DAPI: 3/100 (3.00%)
--------------------------------------------------------------------------------
Stats for marker: Ubiquitin_levels, batch: 2, rep: 1, cell_line: C9orf72-HRE_1008566, condition: stress
Mean: 3673.48, Median: 1328.66, 10th Percentile: 659.91, 50th Percentile (Median): 1328.66, 90th Percentile: 4999.67
Stats for marker: Ubiquitin_levels (All batches, reps, and cell lines)
Mean: 5765.68, Median: 4180.66, 10th Percentile: 1406.83, 50th Percentile (Median): 4180.66, 90th Percentile: 8203.24
Threshold Percentiles (2% - 90%): 661.91 - 8203.24
Filtered Out: 18/100 (18.00%)
Filtered Out_DAPI: 16/100 (16.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 2, rep: 1, cell_line: Control_1017118, condition: stress
Mean: 54449.24, Median: 64084.40, 10th Percentile: 221.07, 50th Percentile (Median): 64084.40, 90th Percentile: 98549.48
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 70/100 (70.00%)
Filtered Out_DAPI: 9/100 (9.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 2, rep: 1, cell_line: Control_1017118, condition: stress
Mean: 47661.43, Median: 36102.11, 10th Percentile: 352.42, 50th Percentile (Median): 36102.11, 90th Percentile: 101714.40
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 86/100 (86.00%)
Filtered Out_DAPI: 9/100 (9.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 2, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 74803.66, Median: 89672.90, 10th Percentile: 25375.82, 50th Percentile (Median): 89672.90, 90th Percentile: 97936.84
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 89/100 (89.00%)
Filtered Out_DAPI: 1/100 (1.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 2, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 61241.55, Median: 82278.91, 10th Percentile: 6375.08, 50th Percentile (Median): 82278.91, 90th Percentile: 102188.02
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 89/100 (89.00%)
Filtered Out_DAPI: 1/100 (1.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 2, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 31289.12, Median: 25727.80, 10th Percentile: 4034.59, 50th Percentile (Median): 25727.80, 90th Percentile: 76519.94
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 49/100 (49.00%)
Filtered Out_DAPI: 65/100 (65.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 2, rep: 1, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 18718.35, Median: 11194.35, 10th Percentile: 2121.93, 50th Percentile (Median): 11194.35, 90th Percentile: 49448.15
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 70/100 (70.00%)
Filtered Out_DAPI: 65/100 (65.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 2, rep: 1, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 24851.24, Median: 8950.50, 10th Percentile: 248.29, 50th Percentile (Median): 8950.50, 90th Percentile: 78042.50
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 39/100 (39.00%)
Filtered Out_DAPI: 3/100 (3.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 2, rep: 1, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 14498.46, Median: 2731.83, 10th Percentile: 852.73, 50th Percentile (Median): 2731.83, 90th Percentile: 58945.06
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 38/100 (38.00%)
Filtered Out_DAPI: 3/100 (3.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 2, rep: 1, cell_line: TDP--43-N390D_1005373, condition: stress
Mean: 27017.36, Median: 21935.59, 10th Percentile: 3824.03, 50th Percentile (Median): 21935.59, 90th Percentile: 63846.69
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 47/100 (47.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 2, rep: 1, cell_line: TDP--43-N390D_1005373, condition: stress
Mean: 11230.68, Median: 8165.40, 10th Percentile: 1360.62, 50th Percentile (Median): 8165.40, 90th Percentile: 26959.74
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 49/100 (49.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: DNA_damage_P53BP1, batch: 2, rep: 2, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 12007.11, Median: 416.05, 10th Percentile: 250.09, 50th Percentile (Median): 416.05, 90th Percentile: 45695.88
Stats for marker: DNA_damage_P53BP1 (All batches, reps, and cell lines)
Mean: 7988.53, Median: 807.94, 10th Percentile: 300.40, 50th Percentile (Median): 807.94, 90th Percentile: 26387.46
Threshold Percentiles (0.3% - 90%): 141.60 - 26387.46
Filtered Out: 19/100 (19.00%)
Filtered Out_DAPI: 5/100 (5.00%)
--------------------------------------------------------------------------------
Stats for marker: Neuronal_activity, batch: 2, rep: 2, cell_line: TDP--43-G348V_1057052, condition: stress
Mean: 10555.90, Median: 2065.12, 10th Percentile: 1095.65, 50th Percentile (Median): 2065.12, 90th Percentile: 26907.30
Stats for marker: Neuronal_activity (All batches, reps, and cell lines)
Mean: 7014.46, Median: 2313.03, 10th Percentile: 961.78, 50th Percentile (Median): 2313.03, 90th Percentile: 8882.09
Threshold Percentiles (5% - 90%): 720.54 - 8882.09
Filtered Out: 25/100 (25.00%)
Filtered Out_DAPI: 5/100 (5.00%)
--------------------------------------------------------------------------------
Stats for marker: Necroptosis_HMGB1, batch: 2, rep: 1, cell_line: C9orf72-HRE_981344, condition: Untreated
Mean: 13866.75, Median: 305.46, 10th Percentile: 192.58, 50th Percentile (Median): 305.46, 90th Percentile: 62556.31
Stats for marker: Necroptosis_HMGB1 (All batches, reps, and cell lines)
Mean: 6658.95, Median: 824.32, 10th Percentile: 326.90, 50th Percentile (Median): 824.32, 90th Percentile: 14553.35
Threshold Percentiles (0.01% - 90%): 116.38 - 14553.35
Filtered Out: 21/100 (21.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: Necrosis, batch: 2, rep: 1, cell_line: C9orf72-HRE_981344, condition: Untreated
Mean: 8763.85, Median: 1061.54, 10th Percentile: 603.71, 50th Percentile (Median): 1061.54, 90th Percentile: 10871.74
Stats for marker: Necrosis (All batches, reps, and cell lines)
Mean: 4499.97, Median: 1645.52, 10th Percentile: 896.73, 50th Percentile (Median): 1645.52, 90th Percentile: 3915.14
Threshold Percentiles (4% - 95%): 636.23 - 7107.17
Filtered Out: 24/100 (24.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: Necroptosis_HMGB1, batch: 2, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 49904.58, Median: 38394.38, 10th Percentile: 10692.39, 50th Percentile (Median): 38394.38, 90th Percentile: 98432.14
Stats for marker: Necroptosis_HMGB1 (All batches, reps, and cell lines)
Mean: 6658.95, Median: 824.32, 10th Percentile: 326.90, 50th Percentile (Median): 824.32, 90th Percentile: 14553.35
Threshold Percentiles (0.01% - 90%): 116.38 - 14553.35
Filtered Out: 79/100 (79.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: Necrosis, batch: 2, rep: 2, cell_line: C9orf72-HRE_981344, condition: stress
Mean: 24736.07, Median: 2939.83, 10th Percentile: 943.57, 50th Percentile (Median): 2939.83, 90th Percentile: 93897.43
Stats for marker: Necrosis (All batches, reps, and cell lines)
Mean: 4499.97, Median: 1645.52, 10th Percentile: 896.73, 50th Percentile (Median): 1645.52, 90th Percentile: 3915.14
Threshold Percentiles (4% - 95%): 636.23 - 7107.17
Filtered Out: 39/100 (39.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: Necroptosis_HMGB1, batch: 2, rep: 2, cell_line: Control_1001733, condition: stress
Mean: 55276.97, Median: 62715.87, 10th Percentile: 2903.11, 50th Percentile (Median): 62715.87, 90th Percentile: 98433.42
Stats for marker: Necroptosis_HMGB1 (All batches, reps, and cell lines)
Mean: 6658.95, Median: 824.32, 10th Percentile: 326.90, 50th Percentile (Median): 824.32, 90th Percentile: 14553.35
Threshold Percentiles (0.01% - 90%): 116.38 - 14553.35
Filtered Out: 76/100 (76.00%)
Filtered Out_DAPI: 1/100 (1.00%)
--------------------------------------------------------------------------------
Stats for marker: Necrosis, batch: 2, rep: 2, cell_line: Control_1001733, condition: stress
Mean: 37068.91, Median: 16906.15, 10th Percentile: 994.81, 50th Percentile (Median): 16906.15, 90th Percentile: 97674.09
Stats for marker: Necrosis (All batches, reps, and cell lines)
Mean: 4499.97, Median: 1645.52, 10th Percentile: 896.73, 50th Percentile (Median): 1645.52, 90th Percentile: 3915.14
Threshold Percentiles (4% - 95%): 636.23 - 7107.17
Filtered Out: 63/100 (63.00%)
Filtered Out_DAPI: 1/100 (1.00%)
--------------------------------------------------------------------------------
Stats for marker: Necroptosis_HMGB1, batch: 2, rep: 2, cell_line: TDP--43-N390D_1005373, condition: stress
Mean: 32283.94, Median: 19431.21, 10th Percentile: 3408.00, 50th Percentile (Median): 19431.21, 90th Percentile: 93524.24
Stats for marker: Necroptosis_HMGB1 (All batches, reps, and cell lines)
Mean: 6658.95, Median: 824.32, 10th Percentile: 326.90, 50th Percentile (Median): 824.32, 90th Percentile: 14553.35
Threshold Percentiles (0.01% - 90%): 116.38 - 14553.35
Filtered Out: 54/100 (54.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------
Stats for marker: Necrosis, batch: 2, rep: 2, cell_line: TDP--43-N390D_1005373, condition: stress
Mean: 17291.72, Median: 1803.69, 10th Percentile: 890.09, 50th Percentile (Median): 1803.69, 90th Percentile: 79014.25
Stats for marker: Necrosis (All batches, reps, and cell lines)
Mean: 4499.97, Median: 1645.52, 10th Percentile: 896.73, 50th Percentile (Median): 1645.52, 90th Percentile: 3915.14
Threshold Percentiles (4% - 95%): 636.23 - 7107.17
Filtered Out: 30/100 (30.00%)
Filtered Out_DAPI: 0/100 (0.00%)
--------------------------------------------------------------------------------


show_images(df_metadata, marker='DNA-damage-pH2Ax', show_DAPI=True, batch=2, cell_line='C9orf72-HRE-1008566', rep=2,
            condition='stress', funova=True, samples=5)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

1
Img shape: (1024, 1024)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f42-ch3t1.tiff
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f42-ch1t1.tiff

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

--------------------------------
2
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)

/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f20-ch3t1.tiff
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f20-ch1t1.tiff
Img shape: (1024, 1024)

WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024).
Cropping image to fit
WARNING:root:Removing pixels from the frame (28, 28)


names = ['Total number of tiles', 'Total number of whole cells']
stats = ['n_valid_tiles','site_whole_cells_counts_sum','site_cell_count','site_cell_count_sum']
total_sum = calc_total_sums(df_target, df_dapi, stats, markers)


plot_marker_data(total_sum, split_by_cell_line=True)

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',


plot_marker_data(total_sum, split_by_cell_line=False)

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1309: FutureWarning: 

The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.

  y='marker',


total_sum.n_valid_tiles.sum()

787463


total_sum[total_sum.marker =='DAPI'].site_whole_cells_counts_sum.sum()

315848.0


total_sum[total_sum.marker =='DAPI'].site_cell_count.sum()

929893.0


show_total_sum_tables(total_sum)


to_heatmap = total_sum.rename(columns={'n_valid_tiles':'index'})
plot_filtering_heatmap(to_heatmap, extra_index='marker', vmin=None, vmax=None,
                      xlabel = 'Total number of tiles', show_sum=True, figsize=(6,24))

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)


to_heatmap = total_sum.rename(columns={'site_whole_cells_counts_sum':'index'})
plot_filtering_heatmap(to_heatmap, extra_index='marker', vmin=None, vmax=None,
                      xlabel = 'Total number of whole cells', show_sum=True, figsize=(6,24))

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)


df_no_empty_sites = df_dapi[df_dapi.n_valid_tiles !=0]
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_cell_count_sum', 
                title='Cell Count Average per Site (from tiles)', figsize=(16,6))

plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_whole_cells_counts_sum',
                title='Whole Cell Count Average per Site',figsize=(16,6))

plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_cell_count',
               title='Cellpose Cell Count Average per Site',figsize=(16,6))


df_dapi.groupby(['cell_line_cond']).n_valid_tiles.mean()

cell_line_cond
C9orf72-HRE-1008566 Untreated      2.450518
C9orf72-HRE-1008566 stress         2.512361
C9orf72-HRE-981344 Untreated       2.192503
C9orf72-HRE-981344 stress          1.274109
Control-1001733 Untreated          4.223693
Control-1001733 stress             3.307299
Control-1017118 Untreated          8.605775
Control-1017118 stress             5.923771
Control-1025045 Untreated          5.488372
Control-1025045 stress             4.039930
Control-1048087 Untreated          4.258272
Control-1048087 stress             3.822094
TDP--43-G348V-1057052 Untreated    1.688427
TDP--43-G348V-1057052 stress       1.205556
TDP--43-N390D-1005373 Untreated    3.614465
TDP--43-N390D-1005373 stress       2.595823
Name: n_valid_tiles, dtype: float64


plot_catplot(df_dapi, sns.color_palette('colorblind'), reps=reps,x='cell_line', y_title='Valid Tiles Count', x_title='Cell Line', y='n_valid_tiles', hue='rep',
             height=4, aspect=2)

/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1028: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  colors_list = custom_palette
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1039: UserWarning: The palette list has more values (10) than needed (2), which may not be intended.
  g.set_axis_labels(x_title, y_title)


# plot_hm(df_dapi, split_by='rep', rows='cell_line', columns='panel', vmax=3)


# for batch in batches:
#     print(batch)
#     run_calc_hist_new(batch,cell_lines_for_disp, markers, root_directory_raw, root_directory_proc,
#                            hist_sample=10,sample_size_per_markers=10, ncols=4, nrows=1, figsize=(6,2))
#     print("="*30)


# save notebook as HTML ( the HTML will be saved in the same folder the original script is)
display(Javascript('IPython.notebook.save_checkpoint();'))
os.system(f'jupyter nbconvert --to html tools/preprocessing_tools/qc_reports/qc_report_funova_exp3.ipynb --output {NOVA_HOME}/manuscript/preprocessing_qc_reports/qc_report_funova_exp3.html')

[NbConvertApp] Converting notebook tools/preprocessing_tools/qc_reports/qc_report_funova_exp3.ipynb to html
[NbConvertApp] Writing 16786962 bytes to /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/manuscript/preprocessing_qc_reports/qc_report_funova_exp3.html

0

	Path	RootFolder	Marker	Condition	CellLine	Batch_Rep	Rep	Batch	Panel	image_id
0	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	stress	TDP--43-G348V-1057052	Batch2/rep1	rep1	Batch2	panelD	r10c07f45-ch2t1
1	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	stress	TDP--43-G348V-1057052	Batch2/rep1	rep1	Batch2	panelD	r10c07f67-ch2t1
2	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	stress	TDP--43-G348V-1057052	Batch2/rep1	rep1	Batch2	panelD	r10c07f87-ch2t1
3	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	stress	TDP--43-G348V-1057052	Batch2/rep1	rep1	Batch2	panelD	r10c07f70-ch2t1
4	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	stress	TDP--43-G348V-1057052	Batch2/rep1	rep1	Batch2	panelD	r10c07f30-ch2t1
...	...	...	...	...	...	...	...	...	...	...
473595	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	DAPI	Untreated	Control-1025045	Batch1/rep2	rep2	Batch1	panelK	r03c22f40-ch1t1
473596	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	DAPI	Untreated	Control-1025045	Batch1/rep2	rep2	Batch1	panelK	r03c22f31-ch1t1
473597	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	DAPI	Untreated	Control-1025045	Batch1/rep2	rep2	Batch1	panelK	r03c22f71-ch1t1
473598	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	DAPI	Untreated	Control-1025045	Batch1/rep2	rep2	Batch1	panelK	r03c22f58-ch1t1
473599	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	DAPI	Untreated	Control-1025045	Batch1/rep2	rep2	Batch1	panelK	r03c22f94-ch1t1

	Path	RootFolder	Marker	Condition	CellLine	Batch_Rep	Rep	Batch	Target_Sharpness_Brenner	Panel	image_id
0	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	Untreated	C9orf72-HRE_1008566	Batch1/rep1	rep1	Batch1	1935.180168	D	r04c07f31-ch2t1
1	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	Untreated	C9orf72-HRE_1008566	Batch1/rep1	rep1	Batch1	611.305103	D	r04c07f82-ch2t1
2	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	Untreated	C9orf72-HRE_1008566	Batch1/rep1	rep1	Batch1	871.004172	D	r04c07f62-ch2t1
3	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	Untreated	C9orf72-HRE_1008566	Batch1/rep1	rep1	Batch1	869.652847	D	r04c07f14-ch2t1
4	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Autophagy	Untreated	C9orf72-HRE_1008566	Batch1/rep1	rep1	Batch1	1258.132105	D	r04c07f11-ch2t1
...	...	...	...	...	...	...	...	...	...	...	...
236795	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Necroptosis_pMLKL	stress	TDP--43-N390D_1005373	Batch2/rep2	rep2	Batch2	16394.368782	K	r14c22f60-ch3t1
236796	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Necroptosis_pMLKL	stress	TDP--43-N390D_1005373	Batch2/rep2	rep2	Batch2	3175.760650	K	r14c22f07-ch3t1
236797	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Necroptosis_pMLKL	stress	TDP--43-N390D_1005373	Batch2/rep2	rep2	Batch2	3673.223571	K	r14c22f55-ch3t1
236798	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Necroptosis_pMLKL	stress	TDP--43-N390D_1005373	Batch2/rep2	rep2	Batch2	3987.187928	K	r14c22f86-ch3t1
236799	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	/home/labs/hornsteinlab/Collaboration/FUNOVA/i...	Necroptosis_pMLKL	stress	TDP--43-N390D_1005373	Batch2/rep2	rep2	Batch2	2342.687136	K	r14c22f02-ch3t1

	n_valid_tiles	% valid tiles	site_whole_cells_counts_sum	site_cell_count
Batch1
count	1181.000000	1181.000000	1181.000000	1.181000e+03
mean	392.540220	3.925402	468.429297	1.370996e+03
std	223.349922	2.233499	295.036673	9.612851e+02
min	2.000000	0.020000	2.000000	2.000000e+00
25%	230.000000	2.300000	254.000000	6.670000e+02
50%	362.000000	3.620000	419.000000	1.169000e+03
75%	531.000000	5.310000	616.000000	1.801000e+03
max	1157.000000	11.570000	1527.000000	4.865000e+03
sum	463590.000000	NaN	553215.000000	1.619146e+06
expected_count	450.000000	450.000000	450.000000	4.500000e+02

	n_valid_tiles	% valid tiles	site_whole_cells_counts_sum	site_cell_count
Batch2
count	1180.000000	1180.000000	1180.000000	1.180000e+03
mean	274.468644	2.744686	332.405932	9.582271e+02
std	215.952885	2.159529	280.652143	9.001070e+02
min	0.000000	0.000000	0.000000	0.000000e+00
25%	114.000000	1.140000	128.000000	3.147500e+02
50%	222.000000	2.220000	253.000000	6.765000e+02
75%	399.000000	3.990000	468.250000	1.376250e+03
max	1198.000000	11.980000	1576.000000	4.868000e+03
sum	323873.000000	NaN	392239.000000	1.130708e+06
expected_count	450.000000	450.000000	450.000000	4.500000e+02

	n valid tiles	% valid tiles	site_whole_cells_counts_sum	site_cell_count
All batches
count	2361.000000	2361.000000	2361.000000	2.361000e+03
mean	333.529437	3.335294	400.446421	1.164699e+03
std	227.436492	2.274365	295.804504	9.536247e+02
min	0.000000	0.000000	0.000000	0.000000e+00
25%	161.000000	1.610000	178.000000	4.510000e+02
50%	292.000000	2.920000	334.000000	9.070000e+02
75%	461.000000	4.610000	553.000000	1.601000e+03
max	1198.000000	11.980000	1576.000000	4.868000e+03
sum	787463.000000	NaN	945454.000000	2.749854e+06
expected_count	450.000000	450.000000	450.000000	4.500000e+02

FUNOVA Preprocessing QC statistics ¶

January 2025¶

Actual Files Validation¶

Raw Files Validation¶

Processed Files Validation¶

Difference between Raw and Processed¶

Variance in each batch (of processed files)¶

Preprocessing Filtering qc¶

1. % site survival after Brenner on DAPI channel¶

2. % Site survival after Cellpose¶

3. % Site survival by tiling¶

4. % Site survival after Brenner on target channel¶

Statistics About the Processed Files¶

Total tiles¶

Total whole nuclei in tiles¶

Total nuclei in sites¶

Show Total Tile Counts¶

Show Total Whole Cell Counts¶

Show Cell Count Statistics per Batch¶

Show Tiles per Site Statistics¶

Show Mean of cell count in valid tiles¶

Assessing Staining Reproducibility and Outliers¶

	Panel	Condition	CellLine	Rep	Batch	Count_table1	Count_table2	Percentage	Combination	Marker
0	J	stress	C9orf72-HRE_981344	rep1	Batch1	221	300	73.666667	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	NaN
1	L	stress	C9orf72-HRE_1008566	rep1	Batch2	285	300	95.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	NaN
2	L	stress	C9orf72-HRE_981344	rep1	Batch2	268	300	89.333333	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	NaN
3	NaN	Untreated	TDP--43-G348V_1057052	rep2	Batch2	84	100	84.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Parthanatos_early
4	NaN	stress	C9orf72-HRE_1008566	rep1	Batch2	96	100	96.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Protein_degradation
5	NaN	stress	C9orf72-HRE_1008566	rep1	Batch2	94	100	94.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Senescence_signaling
6	NaN	stress	C9orf72-HRE_981344	rep1	Batch1	91	100	91.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	DNA_damage_P53BP1
7	NaN	stress	C9orf72-HRE_981344	rep1	Batch1	77	100	77.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Neuronal_activity
8	NaN	stress	C9orf72-HRE_981344	rep1	Batch2	90	100	90.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Protein_degradation
9	NaN	stress	C9orf72-HRE_981344	rep1	Batch2	91	100	91.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Senescence_signaling
10	NaN	stress	C9orf72-HRE_981344	rep2	Batch1	78	100	78.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	Nuclear_speckles_SC35
11	NaN	stress	C9orf72-HRE_981344	rep2	Batch2	79	100	79.000000	['Condition', 'CellLine', 'Rep', 'Batch', 'Mar...	DNA_damage_P53BP1
12	D	stress	Control_1017118	rep1	Batch2	92	100	92.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	DAPI
13	F	stress	C9orf72-HRE_981344	rep2	Batch1	78	100	78.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Nuclear_speckles_SC35
14	H	Untreated	TDP--43-G348V_1057052	rep2	Batch2	84	100	84.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Parthanatos_early
15	J	stress	C9orf72-HRE_981344	rep1	Batch1	91	100	91.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	DNA_damage_P53BP1
16	J	stress	C9orf72-HRE_981344	rep1	Batch1	77	100	77.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Neuronal_activity
17	J	stress	C9orf72-HRE_981344	rep2	Batch2	79	100	79.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	DNA_damage_P53BP1
18	L	stress	C9orf72-HRE_1008566	rep1	Batch2	95	100	95.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	DAPI
19	L	stress	C9orf72-HRE_1008566	rep1	Batch2	96	100	96.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Protein_degradation
20	L	stress	C9orf72-HRE_1008566	rep1	Batch2	94	100	94.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Senescence_signaling
21	L	stress	C9orf72-HRE_981344	rep1	Batch2	87	100	87.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	DAPI
22	L	stress	C9orf72-HRE_981344	rep1	Batch2	90	100	90.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Protein_degradation
23	L	stress	C9orf72-HRE_981344	rep1	Batch2	91	100	91.000000	['Panel', 'Condition', 'CellLine', 'Rep', 'Bat...	Senescence_signaling